import sys
sys.path.append("..")
import scrapy
from scrapy_redis.spiders import RedisCrawlSpider
from redisoper import CRedis

import scrapy
import re
import os
import  urlparse
from items import FruitImgItem
from scrapy import Spider
from scrapy.http import Request


class FruitImageSpilder(Spider):
    redis_key = 'fruitimg_spider:start_urls';
    name = "fruitimgs"

    def make_requests_from_url(self, url):
        query = urlparse.urlparse(url).query
        qs_dict = dict(urlparse.parse_qsl(query))
        fruid=qs_dict['fruidid']
        request= Request(url, dont_filter=True)
        request.meta['fruid']=fruid
        return request;

    def __init__(self):
        self.fruitlistkey = "fruit_list"
        self.r = CRedis()
        self.start_urls =self.initurl();




    def parse(self, response):
        html=response.body
        reg = r'"objURL":"(.*?)"'
        imgre=re.compile(reg)
        print imgre
        imglist=re.findall(imgre,html);
        items = []
        for imgurl in imglist:
            item = FruitImgItem()
            item['image_url']=imgurl;
            item['id'] = response.meta['fruid']
            items.append(item)
        return items
    def initurl(self):
        len = self.r.llen(self.fruitlistkey)

        starturl = []
        for i in range(len):
            fruid = self.r.lindex(self.fruitlistkey,i)
           # path = "/work/scrapy/data/fruitimgs/" + fruid;
            path = "/work/scrapy/data/temp/" + fruid;
            if  os.path.exists(path):
                continue
            v = self.r.get("fruit_" + str(fruid));
            name = v.split(",")[0];

            for j in range(5):
                #url="http://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word="+name+"&ct=201326592&v=flip"
                url="https://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word="+name+"&pn="+str(j*20)+"&gsm=78&fruidid="+str(fruid)
                starturl.append(url)
        return starturl
                #yield scrapy.Request(url, callback=self.parse, meta={'fruitid', fruid})




